# -*- coding: utf-8 -*-
import scrapy
import re
import datetime
import utils
from ip_pool.items import IpPoolItem as Item


class A89ipSpider(scrapy.Spider):
    name = '89ip'
    allowed_domains = ['www.89ip.cn']
    start_urls = ['https://www.89ip.cn/']
    NULL_LIST = []
    ip_set = set()

    def parse(self, response):
        trs = response.css("tbody tr")
        if not trs:
            self.NULL_LIST.append(trs)
        if len(self.NULL_LIST) >= 3:
            return None
        for tr in trs :
            tds = tr.css("td::text").extract()
            item = Item()
            item["from_source"] = self.allowed_domains[0]
            for td in tds:
                print(td)
                td = td.strip().replace("\t", "").replace("\n", "")
                ip = re.match("(([01]{0,1}\d{0,1}\d|2[0-4]\d|25[0-5])\.){3}([01]{0,1}\d{0,1}\d|2[0-4]\d|25[0-5])", td)
                port = re.match("(6[0-5]{2}[0-3][0-5]|[1-5]\d{4}|[1-9]\d{1,3}|[0-9])", td)
                if ip:
                    print(ip)
                    item["ip"] = ip.group()
                    if ip in self.ip_set:
                        break
                    self.ip_set.add(ip)
                    continue
                if port:
                    item["port"] = port.group()
                    break
            if item.get("ip", None):
                item["active"] = True
                item["create_time"] = datetime.datetime.now()
                yield item

        next_url = response.css("a.layui-laypage-next::attr(href)").extract_first()
        next_url = response.urljoin(next_url)
        if next_url:
            print(next_url)
            yield scrapy.Request(next_url, callback=self.parse)

    def delete_89ip_data(self):
        """
        删除89ip在数据库中的数据后，再全量更新
        """
        import pymongo
        url, dbname, collection = utils.generate_mongo_conn()
        myclient = pymongo.MongoClient(url)
        mydb = myclient[dbname]
        mycol = mydb[collection]
        mycol.delete_many({"from_source": self.allowed_domains[0]})